{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM Text Classification with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Loaded Parameters:\n",
      " 40000 35 0.2 200 \n",
      " dataset/glove/glove.twitter.27B.200d.txt\n",
      "[i] Importing Modules...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[+] Using Keras version 2.1.4\n",
      "[i] Finished Importing Modules\n",
      "[i] . Reading from csv file...Done!\n",
      "[i] Found 30505 unique tokens.\n",
      "[+] Shape of data tensor: (40000, 30)\n",
      "[+] Shape of label tensor: (40000, 6)\n",
      "[+] Number of entries in each category:\n",
      "[+] Training:\n",
      " [ 7787.  7450.  4776. 10868.  1036.    83.]\n",
      "[+] Validation:\n",
      " [1857. 1820. 1253. 2756.  287.   27.]\n",
      "[i] Loading GloVe from: dataset/glove/glove.twitter.27B.200d.txt ...Done.\n",
      "[+] Proceeding with Embedding Matrix...Completed!\n",
      "Finished running setup.\n"
     ]
    }
   ],
   "source": [
    "%run Setup.ipynb\n",
    "%run ExtraFunctions.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load embedding matrix into an `Embedding` layer. Toggle `trainable=False` to prevent the weights from being updated during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Structure\n",
    "[Reference](https://github.com/richliao/textClassifier), [LTSM](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "l_lstm1 = Bidirectional(LSTM(14,dropout=0.3,recurrent_dropout=0.3))(embedded_sequences)\n",
    "preds = Dense(6, activation='softmax')(l_lstm1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = Model(sequence_input, preds)\n",
    "adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=adadelta,\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=50, write_grads=True , write_graph=True)\n",
    "model_checkpoints = callbacks.ModelCheckpoint(\"checkpoints\", monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)\n",
    "lr_schedule = callbacks.LearningRateScheduler(initial_boost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -R logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 30)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 30, 200)           6101200   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 28)                24080     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 6)                 174       \n",
      "=================================================================\n",
      "Total params: 6,125,454\n",
      "Trainable params: 24,254\n",
      "Non-trainable params: 6,101,200\n",
      "_________________________________________________________________\n",
      "Training Progress:\n",
      "Train on 32000 samples, validate on 8000 samples\n",
      "Epoch 1/200\n",
      "32000/32000 [==============================] - 23s 729us/step - loss: 1.3688 - acc: 0.4097 - val_loss: 1.2853 - val_acc: 0.4862\n",
      "Epoch 2/200\n",
      "32000/32000 [==============================] - 22s 688us/step - loss: 1.2645 - acc: 0.4864 - val_loss: 1.2614 - val_acc: 0.4999\n",
      "Epoch 3/200\n",
      "32000/32000 [==============================] - 22s 699us/step - loss: 1.2515 - acc: 0.4920 - val_loss: 1.2533 - val_acc: 0.5026\n",
      "Epoch 4/200\n",
      "32000/32000 [==============================] - 22s 694us/step - loss: 1.2427 - acc: 0.4939 - val_loss: 1.2468 - val_acc: 0.5060\n",
      "Epoch 5/200\n",
      "32000/32000 [==============================] - 22s 678us/step - loss: 1.2349 - acc: 0.4990 - val_loss: 1.2438 - val_acc: 0.5090\n",
      "Epoch 6/200\n",
      "32000/32000 [==============================] - 22s 694us/step - loss: 1.2306 - acc: 0.4991 - val_loss: 1.2425 - val_acc: 0.5084\n",
      "Epoch 7/200\n",
      "32000/32000 [==============================] - 22s 691us/step - loss: 1.2213 - acc: 0.5030 - val_loss: 1.2363 - val_acc: 0.5102\n",
      "Epoch 8/200\n",
      "32000/32000 [==============================] - 22s 697us/step - loss: 1.2157 - acc: 0.5042 - val_loss: 1.2345 - val_acc: 0.5107\n",
      "Epoch 9/200\n",
      "32000/32000 [==============================] - 22s 696us/step - loss: 1.2102 - acc: 0.5095 - val_loss: 1.2318 - val_acc: 0.5109\n",
      "Epoch 10/200\n",
      "32000/32000 [==============================] - 22s 674us/step - loss: 1.2068 - acc: 0.5100 - val_loss: 1.2293 - val_acc: 0.5101\n",
      "Epoch 11/200\n",
      "32000/32000 [==============================] - 22s 673us/step - loss: 1.2045 - acc: 0.5114 - val_loss: 1.2292 - val_acc: 0.5084\n",
      "Epoch 12/200\n",
      "32000/32000 [==============================] - 22s 674us/step - loss: 1.1979 - acc: 0.5170 - val_loss: 1.2265 - val_acc: 0.5081\n",
      "Epoch 13/200\n",
      "32000/32000 [==============================] - 22s 675us/step - loss: 1.1944 - acc: 0.5160 - val_loss: 1.2262 - val_acc: 0.5094\n",
      "Epoch 14/200\n",
      "32000/32000 [==============================] - 22s 673us/step - loss: 1.1938 - acc: 0.5194 - val_loss: 1.2268 - val_acc: 0.5075\n",
      "Epoch 15/200\n",
      "32000/32000 [==============================] - 21s 672us/step - loss: 1.1889 - acc: 0.5174 - val_loss: 1.2244 - val_acc: 0.5077\n",
      "Epoch 16/200\n",
      "32000/32000 [==============================] - 22s 679us/step - loss: 1.1860 - acc: 0.5209 - val_loss: 1.2237 - val_acc: 0.5109\n",
      "Epoch 17/200\n",
      "32000/32000 [==============================] - 22s 696us/step - loss: 1.1846 - acc: 0.5205 - val_loss: 1.2258 - val_acc: 0.5094\n",
      "Epoch 18/200\n",
      "32000/32000 [==============================] - 22s 690us/step - loss: 1.1830 - acc: 0.5226 - val_loss: 1.2225 - val_acc: 0.5084\n",
      "Epoch 19/200\n",
      "32000/32000 [==============================] - 22s 693us/step - loss: 1.1813 - acc: 0.5216 - val_loss: 1.2243 - val_acc: 0.5107\n",
      "Epoch 20/200\n",
      "32000/32000 [==============================] - 23s 711us/step - loss: 1.1781 - acc: 0.5238 - val_loss: 1.2225 - val_acc: 0.5085\n",
      "Epoch 21/200\n",
      "32000/32000 [==============================] - 22s 701us/step - loss: 1.1789 - acc: 0.5224 - val_loss: 1.2243 - val_acc: 0.5111\n",
      "Epoch 22/200\n",
      "32000/32000 [==============================] - 22s 684us/step - loss: 1.1741 - acc: 0.5255 - val_loss: 1.2232 - val_acc: 0.5081\n",
      "Epoch 23/200\n",
      "32000/32000 [==============================] - 24s 747us/step - loss: 1.1752 - acc: 0.5286 - val_loss: 1.2254 - val_acc: 0.5075\n",
      "Epoch 24/200\n",
      "32000/32000 [==============================] - 22s 684us/step - loss: 1.1701 - acc: 0.5300 - val_loss: 1.2242 - val_acc: 0.5081\n",
      "Epoch 25/200\n",
      "32000/32000 [==============================] - 21s 671us/step - loss: 1.1691 - acc: 0.5293 - val_loss: 1.2276 - val_acc: 0.5076\n",
      "Epoch 26/200\n",
      "32000/32000 [==============================] - 22s 682us/step - loss: 1.1684 - acc: 0.5312 - val_loss: 1.2265 - val_acc: 0.5085\n",
      "Epoch 27/200\n",
      "32000/32000 [==============================] - 22s 684us/step - loss: 1.1687 - acc: 0.5288 - val_loss: 1.2252 - val_acc: 0.5081\n",
      "Epoch 28/200\n",
      "32000/32000 [==============================] - 22s 693us/step - loss: 1.1625 - acc: 0.5331 - val_loss: 1.2273 - val_acc: 0.5081\n",
      "Epoch 29/200\n",
      "32000/32000 [==============================] - 22s 693us/step - loss: 1.1612 - acc: 0.5293 - val_loss: 1.2277 - val_acc: 0.5075\n",
      "Epoch 30/200\n",
      "32000/32000 [==============================] - 22s 692us/step - loss: 1.1595 - acc: 0.5321 - val_loss: 1.2259 - val_acc: 0.5071\n",
      "Epoch 31/200\n",
      "32000/32000 [==============================] - 22s 695us/step - loss: 1.1625 - acc: 0.5310 - val_loss: 1.2297 - val_acc: 0.5077\n",
      "Epoch 32/200\n",
      "32000/32000 [==============================] - 21s 671us/step - loss: 1.1626 - acc: 0.5317 - val_loss: 1.2277 - val_acc: 0.5091\n",
      "Epoch 33/200\n",
      "32000/32000 [==============================] - 22s 688us/step - loss: 1.1599 - acc: 0.5310 - val_loss: 1.2260 - val_acc: 0.5059\n",
      "Epoch 34/200\n",
      "32000/32000 [==============================] - 22s 680us/step - loss: 1.1595 - acc: 0.5321 - val_loss: 1.2316 - val_acc: 0.5075\n",
      "Epoch 35/200\n",
      "32000/32000 [==============================] - 22s 676us/step - loss: 1.1542 - acc: 0.5346 - val_loss: 1.2277 - val_acc: 0.5040\n",
      "Epoch 36/200\n",
      "32000/32000 [==============================] - 22s 682us/step - loss: 1.1566 - acc: 0.5331 - val_loss: 1.2278 - val_acc: 0.5074\n",
      "Epoch 37/200\n",
      "32000/32000 [==============================] - 23s 704us/step - loss: 1.1529 - acc: 0.5369 - val_loss: 1.2326 - val_acc: 0.5046\n",
      "Epoch 38/200\n",
      "32000/32000 [==============================] - 22s 690us/step - loss: 1.1523 - acc: 0.5385 - val_loss: 1.2266 - val_acc: 0.5049\n",
      "Epoch 39/200\n",
      "32000/32000 [==============================] - 22s 673us/step - loss: 1.1531 - acc: 0.5356 - val_loss: 1.2293 - val_acc: 0.5042\n",
      "Epoch 40/200\n",
      "32000/32000 [==============================] - 22s 673us/step - loss: 1.1504 - acc: 0.5379 - val_loss: 1.2301 - val_acc: 0.5049\n",
      "Epoch 41/200\n",
      "28350/32000 [=========================>....] - ETA: 2s - loss: 1.1512 - acc: 0.5368"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-c0b29fc10fcf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m model_log = model.fit(x_train, y_train, validation_data=(x_val, y_val),\n\u001b[1;32m      5\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m           callbacks=[tensorboard, lr_schedule])\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1710\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1711\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1712\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1713\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1714\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1233\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1236\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2473\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2474\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2475\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2476\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1126\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1127\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1128\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1129\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1130\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1344\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1345\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1346\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1348\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1327\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1328\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1329\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1331\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.summary()\n",
    "model.save('ltsm.h5')\n",
    "print(\"Training Progress:\")\n",
    "model_log = model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          epochs=200, batch_size=50,\n",
    "          callbacks=[tensorboard, lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(model_log.history).to_csv(\"history-inception.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
